# !/usr/bin/python3
# coding: utf-8
# 评论爬取的网址：https://you.ctrip.com/sight/guizhou100064.html

import requests
import json
import re
import time
import pandas as pd

# 转换毫秒时间戳 1.转换成localtime 2.转换成新的时间格式(精确到秒)
def trans(timestamp):
    time_local = time.localtime(int(timestamp) / 1000)
    dt = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
    return dt

view_list = [

     # 景区ID
    ['30805757','天眼基地'],
     #['79837','万峰林'],
     #['80846','梵净山'],
     #['79840','织金洞'],
     #['90599','荔波小七孔'],
     #['82153','西江千户苗寨'],
     #['79796','赤水丹霞'],
     #['80649','青岩古镇'],
     #['79814','黄果树瀑布'],
    # ['79812','龙宫'],
]
for data in view_list:
    id_list = []
    time_list = []
    content_list = []
    name_list = []
    score_list = []
    for x in range(300):
        x = x + 1
        # 存放评论的地址json
        url = "https://m.ctrip.com/restapi/soa2/13444/json/getCommentCollapseList?_fxpcqlniredt=09031026419547123265&x-traceID=09031026419547123265-1660102650846-3627152https://m.ctrip.com/restapi/soa2/13444/json/getCommentCollapseList?_fxpcqlniredt=09031026419547123265&x-traceID=09031026419547123265-1660102650846-3627152"

        payload = {
  "arg": {
    "channelType": 2,
    "collapseType": 0,
    "commentTagId": 0,
    "pageIndex": x,    # 爬取的页数
    "pageSize": 10,
    "poiId": 30805757,   # 景区ID
    "sourceType": 1,
    "sortType": 3,
    "starType": 0
  },
  "head": {
    "cid": "09031026419547123265",
    "ctok": "",
    "cver": "1.0",
    "lang": "01",
    "sid": "8888",
    "syscode": "09",
    "auth": "",
    "xsid": "",
    "extension": []
  }
}

        headers = {
            # 请求头部: 每个景区的cookie不一样，需要手动修改
            'User-Agent': 'Mozilla/5.0(Windows NT 10.0; Win64; x64)AppleWebKit/537.36(KHTML, like Gecko)Chrome/103.0.0.0 Safari/537.36',
            'cookie': 'GUID=09031026419547123265; nfes_isSupportWebP=1; MKT_CKID=1658216345595.jtnp8.vud0; _bfaStatusPVSend=1; _RSG=MuqifwN884CK4pXlZTNMd8; _RGUID=c108217e-a9b3-4e16-89d6-5dd8ef873114; _RDG=28ae4894ab5d76243319aabd6ff660ff33; login_uid=B261680763A645CBE4C2A730C5A44F24; login_type=0; cticket=21CCC3CEB2D03F26982B345BA0F4DD6251D2487143F8EBC098B500FB4E41FBD6; AHeadUserInfo=VipGrade=0&VipGradeName=%C6%D5%CD%A8%BB%E1%D4%B1&UserName=&NoReadMessageCount=0; DUID=u=B261680763A645CBE4C2A730C5A44F24&v=0; IsNonUser=F; IsPersonalizedLogin=F; UUID=0F19DA8CD2EC4CA592995AAA3C96BE87; _ga=GA1.2.263856550.1658731195; appFloatCnt=2; U_TICKET_SELECTED_DISTRICT_CITY={%22value%22:{%22districtid%22:33%2C%22districtname%22:%22%E8%B4%B5%E9%98%B3%22%2C%22isoversea%22:false%2C%22stage%22:%22inipageCity%22}%2C%22updateDate%22:1658731298473%2C%22createTime%22:1658731191846}; MKT_CKID_LMT=1660102583384; _RF1=119.1.160.114; _bfs=1.2; _bfa=1.1658216335331.23eq2w.1.1660102581660.1660111334531.29.189.1; _ubtstatus=%7B%22vid%22%3A%221658216335331.23eq2w%22%2C%22sid%22%3A29%2C%22pvid%22%3A189%2C%22pid%22%3A0%7D; _jzqco=%7C%7C%7C%7C1660102583758%7C1.121496720.1658216345603.1660111335935.1660111342036.1660111335935.1660111342036.undefined.0.0.173.173; __zpspc=9.29.1660111335.1660111342.2%234%7C%7C%7C%7C%7C%23; _bfi=p1%3D290510%26p2%3D290528%26v1%3D189%26v2%3D188; _bfaStatus=success',
            'referer': 'https://you.ctrip.com/',
            'Content-Type': 'application/json;charset=UTF-8',
        }


        response = requests.request("POST", url, headers=headers, data=json.dumps(payload)).text
        html1 = json.loads(response)
      # print(html1)

        comments = html1['result']['items']


        try:
            for i,element in enumerate(comments):
                # print(i,element)
                result = re.findall(r'([0-9]+)',element["publishTime"])
            # print(time[0])
                id_list.append(int())
                time_list.append(trans(result[0]))
                content_list.append(element['content'])
                name_list.append(data[1])
                score_list.append(element['score'])
        except:
            print('{}无数据了'.format(data[1]))
            break
        # print(i,x,element['content'])
        # time = re.findall(r"(.+?)",element["publishTime"])
        # print(i,time)
    id_list = pd.DataFrame(id_list,columns=['id'])
    time_df = pd.DataFrame(time_list,columns=['评论时间'])
    content_df = pd.DataFrame(content_list,columns=['评论内容'])
    name_df = pd.DataFrame(name_list,columns=['景点'])
    score_df = pd.DataFrame(score_list,columns=['用户评分'])
    pd.concat([time_df,name_df,score_df,content_df],axis=1).to_excel('../orginalData/{}数据.xlsx'.format(data[1]))
